# -*- coding: utf-8 -*-
from scpy.logger import get_logger
import os
import sys
from SCAWS import S3

reload(sys)
sys.setdefaultencoding('utf-8')

logger = get_logger(__file__)

CURRENT_PATH = os.path.dirname(__file__)
if CURRENT_PATH:
    CURRENT_PATH = CURRENT_PATH + "/"


def main():
    s3 = S3()
    startMarker = ''
    MAX_NUM = 1000
    BUCKET_NAME = 'c-recruitment'
    KEY_PATH = 'parsed/2016/11/10'
    client = s3._client
    count = 1
    while True:
        # response = client.list_objects(Bucket='recruitment',Delimiter='1',Prefix='src/2016/11/4')

        response = client.list_objects(Bucket=BUCKET_NAME, Prefix=KEY_PATH, MaxKeys=MAX_NUM, Marker=startMarker)
        for item in response.get('Contents',[]):
            key = item.get('Key')
            try:
                with open('/home/scdev/cc/spark-tutorial/cc_data'+key.split('/')[-1], 'wb') as data:
                    client.download_fileobj(BUCKET_NAME, key, data)
                    logger.info('download NO.%d file from aws s3'%(count+1))
            except Exception,e:
                logger.error('produce recruitment info failed for %s'%str(e))
            count += 1
        if response.get('IsTruncated', False):
            startMarker = response.get('Contents')[MAX_NUM-1].get('Key')
        else:
            break


if __name__ == '__main__':
    main()